import java.io.BufferedWriter;
import java.io.FileWriter;
import java.util.*;

public class Main
{
    /**
     * Print differences between analyses in evaluation corpus which have same tag but scored differently.
     */
    public static void CompareAnalyses() throws Throwable
    {
        System.out.println("Parsing Mila");
        Corpus corpusMila = new Corpus("..\\mila");

        for (Article article : corpusMila.articles)
            for (Sentence sentence : article.sentences)
                for (Token token : sentence.tokens)
                    for (int m = 0; m < token.analyses.length; m++)
                        for (int n = m + 1; n < token.analyses.length; n++)
                        {
                            Analysis analysis1 = token.analyses[m];
                            Analysis analysis2 = token.analyses[n];
                            if (analysis1.tag.base.equals(analysis2.tag.base)
                                    && analysis1.score != analysis2.score)
                            {
                                String comp = analysis1.tag.CompareToReference(analysis2.tag);
                                if (comp.contains("suffix") || comp.contains("prefix")
                                        || comp.contains("prefix")
                                        || comp.contains("suffix")
                                        || comp.contains("noun") && comp.contains("definiteness")
                                        || comp.contains("noun") && comp.contains("status")
                                        || comp.contains("adjective") && comp.contains("status")
                                        || comp.contains("adjective") && comp.contains("definiteness")
                                        || comp.contains("properName") && comp.contains("definiteness")
                                        || comp.contains("verb") && comp.contains("tense")
                                        )
                                    continue;

                                String out = "";
                                out += analysis1.parent.GetFullID();
                                out += " " + analysis1.id + " " + analysis2.id;
                                out += " " + comp;
                                System.out.println(out);
                            }
                        }
    }

    /**
     * Print statistics for each article.
     */
    public static void CountAllStats() throws Throwable
    {
        Corpus fullCorpus = new Corpus("../mila");
        for (Article article : fullCorpus.articles)
            System.out.println(article.id + ": " + article.CountStats());
        Dump(0, "Total stats: " + fullCorpus.CountStats());
    }

    /**
     * Compare evaluation corpus against reference corpus.
     * Mark correct analysis in evaluation corpus
     */
    public static void FindErrors() throws Throwable
    {
        BufferedWriter errorWriter = new BufferedWriter(new FileWriter("..\\mila_errors.txt"));

        // change this to 'true' in order to save correct analysis's index
        TextElement.flagModifyDOM = false;
        System.out.println("Parsing Mila");
        Corpus evaluationCorpus = new Corpus("..\\mila");

        System.out.println("Parsing Inbar");
        Corpus referenceCorpus = new Corpus("..\\inbar");

        System.out.println("Comparing Mila and Inbar");

        // Compare all text elements recursively.

        List<Article> evaluationArticles = evaluationCorpus.articles;
        List<Article> referenceArticles = referenceCorpus.articles;
        assert evaluationArticles.size() == referenceArticles.size();

        for (int i = 0; i < evaluationArticles.size(); i++)
        {
            Article evaluationArticle = evaluationArticles.get(i);
            Article referenceArticle = referenceArticles.get(i);
            assert evaluationArticle.id.equals(referenceArticle.id);

            Sentence[] evaluationSentences = evaluationArticle.sentences;
            Sentence[] referenceSentences = referenceArticle.sentences;
            assert evaluationSentences.length == referenceSentences.length;

            for (int j = 0; j < evaluationSentences.length; j++)
            {
                Sentence evaluationSentence = evaluationSentences[j];
                Sentence referenceSentence = referenceSentences[j];
                assert evaluationSentence.id.equals(referenceSentence.id);

                List<Token> evaluationTokens = evaluationSentence.tokens;
                List<Token> referenceTokens = referenceSentence.tokens;
                assert evaluationTokens.size() == referenceTokens.size();

                for (int k = 0; k < evaluationTokens.size(); k++)
                    evaluationTokens.get(k).CompareToReference(referenceTokens.get(k), errorWriter);
            }

            evaluationArticle.SaveXML();

            // Log error ASAP
            errorWriter.flush();
        }

        System.out.println(evaluationCorpus.CountStats());
    }


    static String evaluationCorpusDirName = "d:\\misc\\nlp\\mila";
    static double corpusSplitRatio = 0.9;
    static boolean corpusSplitByErrors = false;
    static Random random = new Random(3);

    /**
     * Stop learning at this score.
     */
    static int minPotentialScore = 4;

    /**
     * Consider only rules which were learned at less this amount of times. 
     */
    static int minRuleCount = 2;

    static boolean useOptimizedAlgorithm = true;
    static int numOfTests = 30;
    static boolean testCorrectness = false;

    public static ScoringFunction scoringFunction =  new LinearScore();

    public static interface ScoringFunction
    {
        double Score(Rule rule, Rule.TestResult result);
    }

    public static class LinearScore implements ScoringFunction
    {
        public double Score(Rule rule, Rule.TestResult result)
        {
            return result.right - result.wrong;
        }

        @Override
        public String toString()
        {
            return "linear";
        }
    }

    public static class LinearLengthScore implements ScoringFunction
    {
        public double Score(Rule rule, Rule.TestResult result)
        {
            return rule.Length() * (result.right - result.wrong);
        }

        @Override
        public String toString()
        {
            return "linear*length";
        }
    }


    public static class LogarithmScore implements ScoringFunction
    {
        public double Score(Rule rule, Rule.TestResult result)
        {
            return (result.right / Math.log(result.wrong + 3) );
        }

        @Override
        public String toString()
        {
            return "logarithm";
        }
    }

    /**
     * Controls which information to output.
     * 0 - minimal output
     * 1 - results after applying each rule
     * 2 - dump rule details
     */
    static int dumpLevel = 0;
    static String separator = "=========================================================\n";

    static void Dump(int level, Object o)
    {
        if (dumpLevel >= level)
            System.out.println(o);
    }

    /**
     * Cross-validation
     */
    public static void RunAllTests() throws Throwable
    {
        // Dump settings
        Dump(0, "Using settings:");
        Dump(0, "Learn partial rules: " + Rule.flagLearnPartialRules);
        Dump(0, "Minimal rule length: " + Rule.minRuleLength);
        Dump(0, "Maximal rule length: " + Rule.maxRuleLength);
        Dump(0, "Corpus split ratio: " + corpusSplitRatio);
        Dump(0, "Using optimized algorithm: " + useOptimizedAlgorithm);



        scoringFunction = new LinearScore();
        minPotentialScore = 2;
        RunTests(numOfTests);

        //scoringFunction = new LinearLengthScore();
        //minPotentialScore = 6;
        //RunTests(numOfTests);
        //
        //scoringFunction = new LogarithmScore();
        //minPotentialScore = 3;
        //RunTests(numOfTests);
    }

    public static void RunTests(int numOfTests) throws Throwable
    {
        for (int testNo = 1; testNo <= numOfTests; testNo++)
        {
            Dump(1, separator + "Running test " + testNo);
            Dump(1, "Loading all articles...");
            Corpus fullCorpus = new Corpus(evaluationCorpusDirName);
            random.setSeed(testNo);
            RunSingleTest(testNo, fullCorpus);
        }
    }

    public static void RunSingleTest(int testNo, Corpus fullCorpus) throws Throwable
    {
        // split corpus into training and evaluation
        Corpus evaluationCorpus = new Corpus();
        Corpus trainingCorpus = new Corpus();
        fullCorpus.Split(trainingCorpus, evaluationCorpus, corpusSplitRatio, random,
                         corpusSplitByErrors);

        Dump(1, "Training statistics: " + trainingCorpus.CountStats());
        Stats startingStats = evaluationCorpus.CountStats();
        Dump(1, "Evaluation statistics: " + startingStats);

        // learn best sequence of rules
        Dump(1, "Training phase");
        List<Utils.Pair<Rule, Rule.TestResult>> chosenRules = LearnRules(trainingCorpus);

        // apply rules to evaluation corpus
        Dump(0, "" + scoringFunction + "\t" + testNo + "\t" + "start" + "\t" + 0 + "\t" + 0 + "\t" + 0
                + "\t" + startingStats.error + "\t" + 0);
        for (int ruleNo = 0; ruleNo < chosenRules.size(); ruleNo++)
        {
            Rule rule = chosenRules.get(ruleNo).first;
            Rule.TestResult learnResult = chosenRules.get(ruleNo).second;
            Rule.TestResult result = evaluationCorpus.ApplyRule(rule, null);
            int currentNumErrors = evaluationCorpus.CountStats().error;
            //Dump(1, result);
            Dump(0, "" + scoringFunction + "\t" + testNo + "\t" + ruleNo + "\t" + rule.Length()
                    + "\t" + result.right + "\t" + result.wrong + "\t" + currentNumErrors
                    + "\t" + scoringFunction.Score(rule, learnResult));
        }
        int finalErrors = evaluationCorpus.CountStats().error;
        Dump(0, "" + scoringFunction + "\t" + testNo + "\t" + "final" + "\t" + 0 + "\t" + 0
                + "\t" + 0 + "\t" + finalErrors + "\t" + 0);
        Dump(1, "Corrected errors: " + (startingStats.error - finalErrors));
        Dump(1, "Number of rules: " + chosenRules.size());
    }



    /**
     * Learn optimal sequence of rules.
     */
    static List<Utils.Pair<Rule, Rule.TestResult>> LearnRules(Corpus trainingCorpus)
    {
        // newly found rules will be appended to end of this list
        List<Utils.Pair<Rule, Rule.TestResult>> chosenRules
                = new LinkedList<Utils.Pair<Rule, Rule.TestResult>>();

        // learn all possible rules
        MultiSet<Rule> allRules = new MultiSet<Rule>();
        trainingCorpus.LearnRules(allRules);
        Dump(1, "Total rules: " + allRules.size());

        // initial filtering
        HashSet<Rule> potentialRules = new HashSet<Rule>();
        if (useOptimizedAlgorithm)
            allRules.filterOutUnique(potentialRules, minRuleCount);
        else
            potentialRules.addAll(allRules);

        // find rules
        Rule rule;
        do
        {
            Dump(1, "Potential rules: " + potentialRules.size());
            HashSet<Rule> rulesToRemove = new HashSet<Rule>();
            rule = FindBestRule(trainingCorpus, potentialRules, rulesToRemove);
            if (useOptimizedAlgorithm)
                potentialRules.removeAll(rulesToRemove);

            if (rule != null) {
                HashSet<Rule> newRules = new HashSet<Rule>();
                Rule.TestResult result = trainingCorpus.ApplyRule(rule, newRules);
                chosenRules.add(new Utils.Pair<Rule, Rule.TestResult>(rule, result));
                potentialRules.addAll(newRules);
                Dump(2, "New rules after applying: " + newRules.size());
                if (useOptimizedAlgorithm)
                {
                    MultiSet<Rule> newRules1 = new MultiSet<Rule>();
                    HashSet<Tag> changedTags = new HashSet<Tag>();
                    for (Rule.Part part : rule.parts)
                        if (part.correctTag != null)
                            changedTags.add(part.originalTag);
                    for (Article article : trainingCorpus.articles)
                        for (Sentence sentence : article.sentences)
                        {
                            HashSet<Integer> changedIndexes = new HashSet<Integer>();
                            for (int index = 0; index < sentence.tokens.size(); index++)
                            {
                                Token token = sentence.tokens.get(index);
                                if (changedTags.contains(token.currentTag))
                                    changedIndexes.add(index);
                            }
                            for (int index : changedIndexes)
                                Rule.LearnAllAroundIndex(newRules1, sentence.tokens, index);
                        }
                    //for (Rule rule1 : newRules1)
                    //    if (newRules1.count(rule1) >= minPotential)
                    //        potentialRules.add(rule1);
                    newRules1.filterOutUnique(potentialRules, minRuleCount);
                    //potentialRules.addAll(newRules);
                }
                else {
                    potentialRules.clear();
                    MultiSet<Rule> newPotentialRules = new MultiSet<Rule>();
                    trainingCorpus.LearnRules(newPotentialRules);
                    newPotentialRules.filterOutUnique(potentialRules, minRuleCount);

                }

                Dump(2, rule);
                Dump(1, "Result: " + result);

                // check that we didn't filter out any good potential rules
                if (testCorrectness)
                {
                    MultiSet<Rule> allRules1 = new MultiSet<Rule>();
                    trainingCorpus.LearnRules(allRules1);
                    HashSet<Rule> allPotentialRules = new HashSet<Rule>();
                    HashSet<Rule> nonPotentialRules = new HashSet<Rule>();
                    allRules1.filterOutUnique(allPotentialRules, minRuleCount);
                    FindBestRule(trainingCorpus, allPotentialRules, nonPotentialRules);
                    allPotentialRules.removeAll(nonPotentialRules);
                    //allPotentialRules.removeAll(potentialRules);
                    //if (allPotentialRules.size() > 0)
                    //{
                    //    //for (Rule rule1 : allPotentialRules)
                    //    //    Dump(0, "" + rule1 + "\n" + trainingCorpus.TestRule(rule1) +
                    //    //            "\n" + trainingCorpus.ApplyRule(rule1, null));
                    //    Dump(0, Utils.join(allPotentialRules, "\n\n"));
                    //    Dump(0, "New rules:");
                    //    Dump(0, Utils.join(newRules, "\n\n"));
                    //}
                    assert potentialRules.containsAll(allPotentialRules);
                }
            }
        }
        while (rule != null);

        return chosenRules;
    }

    /**
     * Find a rule which corrects maximal number of errors.
     * @param rules - rules to choose from
     * @param nonPotentialRules - if not null, put rules without potential here
     */
    public static Rule FindBestRule(Corpus corpus, Collection<Rule> rules,
                                    Collection<Rule> nonPotentialRules)
    {
        //Rule.TestResult bestResult = new Rule.TestResult();
        Rule bestRule = null;
        double bestScore = minPotentialScore;
        for (Rule rule : rules)
        {
            Rule.TestResult result = corpus.TestRule(rule);
            if (scoringFunction.Score(rule, result) < minPotentialScore)
            {
                if (nonPotentialRules != null)
                    nonPotentialRules.add(rule);
            }
            else if (scoringFunction.Score(rule, result) >= bestScore) {
                bestRule = rule;
                //bestResult = result;
                bestScore = scoringFunction.Score(rule, result);
            }
        }
        return bestRule;
    }

    public static void main(String[] argv) throws Throwable
    {
        // uncomment needed routine

        RunAllTests();
        //CountAllStats();
        //FindErrors();
        //CompareAnalyses();
    }
}
